Personal Computer World 2009 February/PCWFEB09.iso/Software/Resources/Chat & Communication/Digsby build 37/digsby_setup.exe/lib/urllib2.pyo

Python Compiled Bytecode | 2008-10-13 | 35KB | 1,174 lines

# Source Generated with Decompyle++
# File: in.pyo (Python 2.5)

import base64
import hashlib
import httplib
import mimetools
import os
import posixpath
import random
import re
import socket
import sys
import time
import urlparse
import bisect

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from urllib import unwrap, unquote, splittype, splithost, quote, addinfourl, splitport, splitgophertype, splitquery, splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue
from urllib import localhost, url2pathname, getproxies
__version__ = sys.version[:3]
_opener = None

def urlopen(url, data = None):
    global _opener
    if _opener is None:
        _opener = build_opener()

    return _opener.open(url, data)


def install_opener(opener):
    global _opener
    _opener = opener


class URLError(IOError):

    def __init__(self, reason):
        self.args = (reason,)
        self.reason = reason


    def __str__(self):
        return '<urlopen error %s>' % self.reason



class HTTPError(URLError, addinfourl):
    __super_init = addinfourl.__init__

    def __init__(self, url, code, msg, hdrs, fp):
        self.code = code
        self.msg = msg
        self.hdrs = hdrs
        self.fp = fp
        self.filename = url
        if fp is not None:
            self._HTTPError__super_init(fp, hdrs, url)



    def __str__(self):
        return 'HTTP Error %s: %s' % (self.code, self.msg)



class GopherError(URLError):
    pass

_cut_port_re = re.compile(':\\d+$')

def request_host(request):
    url = request.get_full_url()
    host = urlparse.urlparse(url)[1]
    if host == '':
        host = request.get_header('Host', '')

    host = _cut_port_re.sub('', host, 1)
    return host.lower()


class Request:

    def __init__(self, url, data = None, headers = { }, origin_req_host = None, unverifiable = False):
        self._Request__original = unwrap(url)
        self.type = None
        self.host = None
        self.port = None
        self.data = data
        self.headers = { }
        for key, value in headers.items():
            self.add_header(key, value)

        self.unredirected_hdrs = { }
        if origin_req_host is None:
            origin_req_host = request_host(self)

        self.origin_req_host = origin_req_host
        self.unverifiable = unverifiable


    def __getattr__(self, attr):
        if attr[:12] == '_Request__r_':
            name = attr[12:]
            if hasattr(Request, 'get_' + name):
                getattr(self, 'get_' + name)()
                return getattr(self, attr)


        raise AttributeError, attr


    def get_method(self):
        if self.has_data():
            return 'POST'
        else:
            return 'GET'


    def add_data(self, data):
        self.data = data


    def has_data(self):
        return self.data is not None


    def get_data(self):
        return self.data


    def get_full_url(self):
        return self._Request__original


    def get_type(self):
        if self.type is None:
            (self.type, self._Request__r_type) = splittype(self._Request__original)
            if self.type is None:
                raise ValueError, 'unknown url type: %s' % self._Request__original


        return self.type


    def get_host(self):
        if self.host is None:
            (self.host, self._Request__r_host) = splithost(self._Request__r_type)
            if self.host:
                self.host = unquote(self.host)


        return self.host


    def get_selector(self):
        return self._Request__r_host


    def set_proxy(self, host, type):
        self.host = host
        self.type = type
        self._Request__r_host = self._Request__original


    def get_origin_req_host(self):
        return self.origin_req_host


    def is_unverifiable(self):
        return self.unverifiable


    def add_header(self, key, val):
        self.headers[key.capitalize()] = val


    def add_unredirected_header(self, key, val):
        self.unredirected_hdrs[key.capitalize()] = val


    def has_header(self, header_name):
        if not header_name in self.headers:
            pass
        return header_name in self.unredirected_hdrs


    def get_header(self, header_name, default = None):
        return self.headers.get(header_name, self.unredirected_hdrs.get(header_name, default))


    def header_items(self):
        hdrs = self.unredirected_hdrs.copy()
        hdrs.update(self.headers)
        return hdrs.items()



class OpenerDirector:

    def __init__(self):
        client_version = 'Python-urllib/%s' % __version__
        self.addheaders = [
            ('User-agent', client_version)]
        self.handlers = []
        self.handle_open = { }
        self.handle_error = { }
        self.process_response = { }
        self.process_request = { }


    def add_handler(self, handler):
        if not hasattr(handler, 'add_parent'):
            raise TypeError('expected BaseHandler instance, got %r' % type(handler))

        added = False
        for meth in dir(handler):
            if meth in ('redirect_request', 'do_open', 'proxy_open'):
                continue

            i = meth.find('_')
            protocol = meth[:i]
            condition = meth[i + 1:]
            if condition.startswith('error'):
                j = condition.find('_') + i + 1
                kind = meth[j + 1:]

                try:
                    kind = int(kind)
                except ValueError:
                    pass

                lookup = self.handle_error.get(protocol, { })
                self.handle_error[protocol] = lookup
            elif condition == 'open':
                kind = protocol
                lookup = self.handle_open
            elif condition == 'response':
                kind = protocol
                lookup = self.process_response
            elif condition == 'request':
                kind = protocol
                lookup = self.process_request

            handlers = lookup.setdefault(kind, [])
            if handlers:
                bisect.insort(handlers, handler)
            else:
                handlers.append(handler)
            added = True

        if added:
            bisect.insort(self.handlers, handler)
            handler.add_parent(self)



    def close(self):
        pass


    def _call_chain(self, chain, kind, meth_name, *args):
        handlers = chain.get(kind, ())
        for handler in handlers:
            func = getattr(handler, meth_name)
            result = func(*args)
            if result is not None:
                return result
                continue



    def open(self, fullurl, data = None):
        if isinstance(fullurl, basestring):
            req = Request(fullurl, data)
        else:
            req = fullurl
            if data is not None:
                req.add_data(data)

        protocol = req.get_type()
        meth_name = protocol + '_request'
        for processor in self.process_request.get(protocol, []):
            meth = getattr(processor, meth_name)
            req = meth(req)

        response = self._open(req, data)
        meth_name = protocol + '_response'
        for processor in self.process_response.get(protocol, []):
            meth = getattr(processor, meth_name)
            response = meth(req, response)

        return response


    def _open(self, req, data = None):
        result = self._call_chain(self.handle_open, 'default', 'default_open', req)
        if result:
            return result

        protocol = req.get_type()
        result = self._call_chain(self.handle_open, protocol, protocol + '_open', req)
        if result:
            return result

        return self._call_chain(self.handle_open, 'unknown', 'unknown_open', req)


    def error(self, proto, *args):
        if proto in ('http', 'https'):
            dict = self.handle_error['http']
            proto = args[2]
            meth_name = 'http_error_%s' % proto
            http_err = 1
            orig_args = args
        else:
            dict = self.handle_error
            meth_name = proto + '_error'
            http_err = 0
        args = (dict, proto, meth_name) + args
        result = self._call_chain(*args)
        if result:
            return result

        if http_err:
            args = (dict, 'default', 'http_error_default') + orig_args
            return self._call_chain(*args)




def build_opener(*handlers):
    import types

    def isclass(obj):
        if not isinstance(obj, types.ClassType):
            pass
        return hasattr(obj, '__bases__')

    opener = OpenerDirector()
    default_classes = [
        ProxyHandler,
        UnknownHandler,
        HTTPHandler,
        HTTPDefaultErrorHandler,
        HTTPRedirectHandler,
        FTPHandler,
        FileHandler,
        HTTPErrorProcessor]
    if hasattr(httplib, 'HTTPS'):
        default_classes.append(HTTPSHandler)

    skip = set()
    for klass in default_classes:
        for check in handlers:
            if isclass(check):
                if issubclass(check, klass):
                    skip.add(klass)

            issubclass(check, klass)
            if isinstance(check, klass):
                skip.add(klass)
                continue


    for klass in skip:
        default_classes.remove(klass)

    for klass in default_classes:
        opener.add_handler(klass())

    for h in handlers:
        if isclass(h):
            h = h()

        opener.add_handler(h)

    return opener


class BaseHandler:
    handler_order = 500

    def add_parent(self, parent):
        self.parent = parent


    def close(self):
        pass


    def __lt__(self, other):
        if not hasattr(other, 'handler_order'):
            return True

        return self.handler_order < other.handler_order



class HTTPErrorProcessor(BaseHandler):
    handler_order = 1000

    def http_response(self, request, response):
        code = response.code
        msg = response.msg
        hdrs = response.info()
        if code not in (200, 206):
            response = self.parent.error('http', request, response, code, msg, hdrs)

        return response

    https_response = http_response


class HTTPDefaultErrorHandler(BaseHandler):

    def http_error_default(self, req, fp, code, msg, hdrs):
        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)



class HTTPRedirectHandler(BaseHandler):
    max_repeats = 4
    max_redirections = 10

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        m = req.get_method()
        if (code in (301, 302, 303, 307) or m in ('GET', 'HEAD') or code in (301, 302, 303)) and m == 'POST':
            newurl = newurl.replace(' ', '%20')
            return Request(newurl, headers = req.headers, origin_req_host = req.get_origin_req_host(), unverifiable = True)
        else:
            raise HTTPError(req.get_full_url(), code, msg, headers, fp)


    def http_error_302(self, req, fp, code, msg, headers):
        if 'location' in headers:
            newurl = headers.getheaders('location')[0]
        elif 'uri' in headers:
            newurl = headers.getheaders('uri')[0]
        else:
            return None
        newurl = urlparse.urljoin(req.get_full_url(), newurl)
        new = self.redirect_request(req, fp, code, msg, headers, newurl)
        if new is None:
            return None

        if hasattr(req, 'redirect_dict'):
            visited = new.redirect_dict = req.redirect_dict
            if visited.get(newurl, 0) >= self.max_repeats or len(visited) >= self.max_redirections:
                raise HTTPError(req.get_full_url(), code, self.inf_msg + msg, headers, fp)

        else:
            visited = new.redirect_dict = req.redirect_dict = { }
        visited[newurl] = visited.get(newurl, 0) + 1
        fp.read()
        fp.close()
        return self.parent.open(new)

    http_error_301 = http_error_303 = http_error_307 = http_error_302
    inf_msg = 'The HTTP server returned a redirect error that would lead to an infinite loop.\nThe last 30x error message was:\n'


def _parse_proxy(proxy):
    (scheme, r_scheme) = splittype(proxy)
    if not r_scheme.startswith('/'):
        scheme = None
        authority = proxy
    elif not r_scheme.startswith('//'):
        raise ValueError('proxy URL with no authority: %r' % proxy)

    end = r_scheme.find('/', 2)
    if end == -1:
        end = None

    authority = r_scheme[2:end]
    (userinfo, hostport) = splituser(authority)
    if userinfo is not None:
        (user, password) = splitpasswd(userinfo)
    else:
        user = None
        password = None
    return (scheme, user, password, hostport)


class ProxyHandler(BaseHandler):
    handler_order = 100

    def __init__(self, proxies = None):
        if proxies is None:
            proxies = getproxies()

        self.proxies = proxies
        for type, url in proxies.items():
            setattr(self, '%s_open' % type, (lambda r, proxy = url, type = type, meth = self.proxy_open: meth(r, proxy, type)))



    def proxy_open(self, req, proxy, type):
        orig_type = req.get_type()
        (proxy_type, user, password, hostport) = _parse_proxy(proxy)
        if proxy_type is None:
            proxy_type = orig_type

        if user and password:
            user_pass = '%s:%s' % (unquote(user), unquote(password))
            creds = base64.b64encode(user_pass).strip()
            req.add_header('Proxy-authorization', 'Basic ' + creds)

        hostport = unquote(hostport)
        req.set_proxy(hostport, proxy_type)
        if orig_type == proxy_type:
            return None
        else:
            return self.parent.open(req)



class HTTPPasswordMgr:

    def __init__(self):
        self.passwd = { }


    def add_password(self, realm, uri, user, passwd):
        if isinstance(uri, basestring):
            uri = [
                uri]

        if realm not in self.passwd:
            self.passwd[realm] = { }

        for default_port in (True, False):
            reduced_uri = []([ self.reduce_uri(u, default_port) for u in uri ])
            self.passwd[realm][reduced_uri] = (user, passwd)



    def find_user_password(self, realm, authuri):
        domains = self.passwd.get(realm, { })
        for default_port in (True, False):
            reduced_authuri = self.reduce_uri(authuri, default_port)
            for uris, authinfo in domains.iteritems():
                for uri in uris:
                    if self.is_suburi(uri, reduced_authuri):
                        return authinfo
                        continue



        return (None, None)


    def reduce_uri(self, uri, default_port = True):
        parts = urlparse.urlsplit(uri)
        if parts[1]:
            scheme = parts[0]
            authority = parts[1]
            if not parts[2]:
                pass
            path = '/'
        else:
            scheme = None
            authority = uri
            path = '/'
        (host, port) = splitport(authority)
        if default_port and port is None and scheme is not None:
            dport = {
                'http': 80,
                'https': 443 }.get(scheme)
            if dport is not None:
                authority = '%s:%d' % (host, dport)


        return (authority, path)


    def is_suburi(self, base, test):
        if base == test:
            return True

        if base[0] != test[0]:
            return False

        common = posixpath.commonprefix((base[1], test[1]))
        if len(common) == len(base[1]):
            return True

        return False



class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):

    def find_user_password(self, realm, authuri):
        (user, password) = HTTPPasswordMgr.find_user_password(self, realm, authuri)
        if user is not None:
            return (user, password)

        return HTTPPasswordMgr.find_user_password(self, None, authuri)



class AbstractBasicAuthHandler:
    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', re.I)

    def __init__(self, password_mgr = None):
        if password_mgr is None:
            password_mgr = HTTPPasswordMgr()

        self.passwd = password_mgr
        self.add_password = self.passwd.add_password


    def http_error_auth_reqed(self, authreq, host, req, headers):
        authreq = headers.get(authreq, None)
        if authreq:
            mo = AbstractBasicAuthHandler.rx.search(authreq)
            if mo:
                (scheme, realm) = mo.groups()
                if scheme.lower() == 'basic':
                    return self.retry_http_basic_auth(host, req, realm)





    def retry_http_basic_auth(self, host, req, realm):
        (user, pw) = self.passwd.find_user_password(realm, host)
        if pw is not None:
            raw = '%s:%s' % (user, pw)
            auth = 'Basic %s' % base64.b64encode(raw).strip()
            if req.headers.get(self.auth_header, None) == auth:
                return None

            req.add_header(self.auth_header, auth)
            return self.parent.open(req)
        else:
            return None



class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
    auth_header = 'Authorization'

    def http_error_401(self, req, fp, code, msg, headers):
        url = req.get_full_url()
        return self.http_error_auth_reqed('www-authenticate', url, req, headers)



class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
    auth_header = 'Proxy-authorization'

    def http_error_407(self, req, fp, code, msg, headers):
        authority = req.get_host()
        return self.http_error_auth_reqed('proxy-authenticate', authority, req, headers)



def randombytes(n):
    pass


class AbstractDigestAuthHandler:

    def __init__(self, passwd = None):
        if passwd is None:
            passwd = HTTPPasswordMgr()

        self.passwd = passwd
        self.add_password = self.passwd.add_password
        self.retried = 0
        self.nonce_count = 0


    def reset_retry_count(self):
        self.retried = 0


    def http_error_auth_reqed(self, auth_header, host, req, headers):
        authreq = headers.get(auth_header, None)
        if authreq:
            scheme = authreq.split()[0]
            if scheme.lower() == 'digest':
                return self.retry_http_digest_auth(req, authreq)




    def retry_http_digest_auth(self, req, auth):
        (token, challenge) = auth.split(' ', 1)
        chal = parse_keqv_list(parse_http_list(challenge))
        auth = self.get_authorization(req, chal)
        if auth:
            auth_val = 'Digest %s' % auth
            if req.headers.get(self.auth_header, None) == auth_val:
                return None

            req.add_unredirected_header(self.auth_header, auth_val)
            resp = self.parent.open(req)
            return resp



    def get_cnonce(self, nonce):
        dig = hashlib.sha1('%s:%s:%s:%s' % (self.nonce_count, nonce, time.ctime(), randombytes(8))).hexdigest()
        return dig[:16]


    def get_authorization(self, req, chal):

        try:
            realm = chal['realm']
            nonce = chal['nonce']
            qop = chal.get('qop')
            algorithm = chal.get('algorithm', 'MD5')
            opaque = chal.get('opaque', None)
        except KeyError:
            return None

        (H, KD) = self.get_algorithm_impls(algorithm)
        if H is None:
            return None

        (user, pw) = self.passwd.find_user_password(realm, req.get_full_url())
        if user is None:
            return None

        if req.has_data():
            entdig = self.get_entity_digest(req.get_data(), chal)
        else:
            entdig = None
        A1 = '%s:%s:%s' % (user, realm, pw)
        A2 = '%s:%s' % (req.get_method(), req.get_selector())
        if qop == 'auth':
            self.nonce_count += 1
            ncvalue = '%08x' % self.nonce_count
            cnonce = self.get_cnonce(nonce)
            noncebit = '%s:%s:%s:%s:%s' % (nonce, ncvalue, cnonce, qop, H(A2))
            respdig = KD(H(A1), noncebit)
        elif qop is None:
            respdig = KD(H(A1), '%s:%s' % (nonce, H(A2)))
        else:
            raise URLError("qop '%s' is not supported." % qop)
        base = 'username="%s", realm="%s", nonce="%s", uri="%s", response="%s"' % (user, realm, nonce, req.get_selector(), respdig)
        if opaque:
            base += ', opaque="%s"' % opaque

        if entdig:
            base += ', digest="%s"' % entdig

        base += ', algorithm="%s"' % algorithm
        if qop:
            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)

        return base


    def get_algorithm_impls(self, algorithm):
        if algorithm == 'MD5':

            H = lambda x: hashlib.md5(x).hexdigest()
        elif algorithm == 'SHA':

            H = lambda x: hashlib.sha1(x).hexdigest()


        KD = lambda s, d: H('%s:%s' % (s, d))
        return (H, KD)


    def get_entity_digest(self, data, chal):
        pass



class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
    auth_header = 'Authorization'
    handler_order = 490

    def http_error_401(self, req, fp, code, msg, headers):
        host = urlparse.urlparse(req.get_full_url())[1]
        retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
        self.reset_retry_count()
        return retry



class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
    auth_header = 'Proxy-Authorization'
    handler_order = 490

    def http_error_407(self, req, fp, code, msg, headers):
        host = req.get_host()
        retry = self.http_error_auth_reqed('proxy-authenticate', host, req, headers)
        self.reset_retry_count()
        return retry



class AbstractHTTPHandler(BaseHandler):

    def __init__(self, debuglevel = 0):
        self._debuglevel = debuglevel


    def set_http_debuglevel(self, level):
        self._debuglevel = level


    def do_request_(self, request):
        host = request.get_host()
        if not host:
            raise URLError('no host given')

        if request.has_data():
            data = request.get_data()
            if not request.has_header('Content-type'):
                request.add_unredirected_header('Content-type', 'application/x-www-form-urlencoded')

            if not request.has_header('Content-length'):
                request.add_unredirected_header('Content-length', '%d' % len(data))


        (scheme, sel) = splittype(request.get_selector())
        (sel_host, sel_path) = splithost(sel)
        if not request.has_header('Host'):
            if not sel_host:
                pass
            request.add_unredirected_header('Host', host)

        for name, value in self.parent.addheaders:
            name = name.capitalize()
            if not request.has_header(name):
                request.add_unredirected_header(name, value)
                continue

        return request


    def do_open(self, http_class, req):
        host = req.get_host()
        if not host:
            raise URLError('no host given')

        h = http_class(host)
        h.set_debuglevel(self._debuglevel)
        headers = dict(req.headers)
        headers.update(req.unredirected_hdrs)
        headers['Connection'] = 'close'
        headers = dict((lambda .0: for name, val in .0:
(name.title(), val))(headers.items()))

        try:
            h.request(req.get_method(), req.get_selector(), req.data, headers)
            r = h.getresponse()
        except socket.error:
            err = None
            raise URLError(err)

        r.recv = r.read
        fp = socket._fileobject(r, close = True)
        resp = addinfourl(fp, r.msg, req.get_full_url())
        resp.code = r.status
        resp.msg = r.reason
        return resp



class HTTPHandler(AbstractHTTPHandler):

    def http_open(self, req):
        return self.do_open(httplib.HTTPConnection, req)

    http_request = AbstractHTTPHandler.do_request_

if hasattr(httplib, 'HTTPS'):

    class HTTPSHandler(AbstractHTTPHandler):

        def https_open(self, req):
            return self.do_open(httplib.HTTPSConnection, req)

        https_request = AbstractHTTPHandler.do_request_



class HTTPCookieProcessor(BaseHandler):

    def __init__(self, cookiejar = None):
        import cookielib as cookielib
        if cookiejar is None:
            cookiejar = cookielib.CookieJar()

        self.cookiejar = cookiejar


    def http_request(self, request):
        self.cookiejar.add_cookie_header(request)
        return request


    def http_response(self, request, response):
        self.cookiejar.extract_cookies(response, request)
        return response

    https_request = http_request
    https_response = http_response


class UnknownHandler(BaseHandler):

    def unknown_open(self, req):
        type = req.get_type()
        raise URLError('unknown url type: %s' % type)



def parse_keqv_list(l):
    parsed = { }
    for elt in l:
        (k, v) = elt.split('=', 1)
        if v[0] == '"' and v[-1] == '"':
            v = v[1:-1]

        parsed[k] = v

    return parsed


def parse_http_list(s):
    res = []
    part = ''
    escape = quote = False
    for cur in s:
        if escape:
            part += cur
            escape = False
            continue

        if quote:
            if cur == '\\':
                escape = True
                continue
            elif cur == '"':
                quote = False

            part += cur
            continue

        if cur == ',':
            res.append(part)
            part = ''
            continue

        if cur == '"':
            quote = True

        part += cur

    if part:
        res.append(part)

    return [ part.strip() for part in res ]


class FileHandler(BaseHandler):

    def file_open(self, req):
        url = req.get_selector()
        if url[:2] == '//' and url[2:3] != '/':
            req.type = 'ftp'
            return self.parent.open(req)
        else:
            return self.open_local_file(req)

    names = None

    def get_names(self):
        if FileHandler.names is None:

            try:
                FileHandler.names = (socket.gethostbyname('localhost'), socket.gethostbyname(socket.gethostname()))
            except socket.gaierror:
                FileHandler.names = (socket.gethostbyname('localhost'),)
            except:
                None<EXCEPTION MATCH>socket.gaierror


        None<EXCEPTION MATCH>socket.gaierror
        return FileHandler.names


    def open_local_file(self, req):
        import email.Utils as email
        import mimetypes as mimetypes
        host = req.get_host()
        file = req.get_selector()
        localfile = url2pathname(file)
        stats = os.stat(localfile)
        size = stats.st_size
        modified = email.Utils.formatdate(stats.st_mtime, usegmt = True)
        mtype = mimetypes.guess_type(file)[0]
        if not mtype:
            pass
        headers = mimetools.Message(StringIO('Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % ('text/plain', size, modified)))
        if host:
            (host, port) = splitport(host)

        if (not host or not port) and socket.gethostbyname(host) in self.get_names():
            return addinfourl(open(localfile, 'rb'), headers, 'file:' + file)

        raise URLError('file not on local host')



class FTPHandler(BaseHandler):

    def ftp_open(self, req):
        import ftplib as ftplib
        import mimetypes
        host = req.get_host()
        if not host:
            raise IOError, ('ftp error', 'no host given')

        (host, port) = splitport(host)
        if port is None:
            port = ftplib.FTP_PORT
        else:
            port = int(port)
        (user, host) = splituser(host)
        if user:
            (user, passwd) = splitpasswd(user)
        else:
            passwd = None
        host = unquote(host)
        if not user:
            pass
        user = unquote('')
        if not passwd:
            pass
        passwd = unquote('')

        try:
            host = socket.gethostbyname(host)
        except socket.error:
            msg = None
            raise URLError(msg)

        (path, attrs) = splitattr(req.get_selector())
        dirs = path.split('/')
        dirs = map(unquote, dirs)
        dirs = dirs[:-1]
        file = dirs[-1]
        if dirs and not dirs[0]:
            dirs = dirs[1:]


        try:
            fw = self.connect_ftp(user, passwd, host, port, dirs)
            if not file or 'I':
                pass
            type = 'D'
            for attr in attrs:
                (attr, value) = splitvalue(attr)
                if attr.lower() == 'type' and value in ('a', 'A', 'i', 'I', 'd', 'D'):
                    type = value.upper()
                    continue

            (fp, retrlen) = fw.retrfile(file, type)
            headers = ''
            mtype = mimetypes.guess_type(req.get_full_url())[0]
            if mtype:
                headers += 'Content-type: %s\n' % mtype

            if retrlen is not None and retrlen >= 0:
                headers += 'Content-length: %d\n' % retrlen

            sf = StringIO(headers)
            headers = mimetools.Message(sf)
            return addinfourl(fp, headers, req.get_full_url())
        except ftplib.all_errors:
            msg = None
            raise IOError, ('ftp error', msg), sys.exc_info()[2]



    def connect_ftp(self, user, passwd, host, port, dirs):
        fw = ftpwrapper(user, passwd, host, port, dirs)
        return fw



class CacheFTPHandler(FTPHandler):

    def __init__(self):
        self.cache = { }
        self.timeout = { }
        self.soonest = 0
        self.delay = 60
        self.max_conns = 16


    def setTimeout(self, t):
        self.delay = t


    def setMaxConns(self, m):
        self.max_conns = m


    def connect_ftp(self, user, passwd, host, port, dirs):
        key = (user, host, port, '/'.join(dirs))
        if key in self.cache:
            self.timeout[key] = time.time() + self.delay
        else:
            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs)
            self.timeout[key] = time.time() + self.delay
        self.check_cache()
        return self.cache[key]


    def check_cache(self):
        t = time.time()
        if self.soonest <= t:
            for k, v in self.timeout.items():
                if v < t:
                    self.cache[k].close()
                    del self.cache[k]
                    del self.timeout[k]
                    continue


        self.soonest = min(self.timeout.values())
        if len(self.cache) == self.max_conns:
            for k, v in self.timeout.items():
                if v == self.soonest:
                    del self.cache[k]
                    del self.timeout[k]
                    break
                    continue

            self.soonest = min(self.timeout.values())




class GopherHandler(BaseHandler):

    def gopher_open(self, req):
        import gopherlib as gopherlib
        host = req.get_host()
        if not host:
            raise GopherError('no host given')

        host = unquote(host)
        selector = req.get_selector()
        (type, selector) = splitgophertype(selector)
        (selector, query) = splitquery(selector)
        selector = unquote(selector)
        if query:
            query = unquote(query)
            fp = gopherlib.send_query(selector, query, host)
        else:
            fp = gopherlib.send_selector(selector, host)
        return addinfourl(fp, noheaders(), req.get_full_url())